#!/usr/bin/python

# visualize average CPU (mem, I/O) usage of specific job on each cluster machine
# compared with average CPU (mem, I/O) usage per machine

# CPU usage : %
# rbytes: KB/s
# wbytes: KB/s
# virtual mem: MB
# physical mem: MB

import sys
import re


def getvalue(a):
	return a[a.find("=")+1:].strip("\"")

def getjobid(a):
	A = a.split('_')
	if len(A) == 1:
		return "99999999"
	else:
		return A[2]

def gettype(a):
	if len(A) == 1:
		return A[0]
	else:
		return "notype"


if len(sys.argv) <= 2:
	print "./COMMAND [job tracker log file name] [machine number]?\n"
	sys.exit(-1)


# get per machine active time information
hosts = {}
for l in open("time.txt"):
	A = l.split()
	hosts[int(A[0])]= {}
	hosts[int(A[0])]["start"] = int(A[1])
	hosts[int(A[0])]["finish"] = int(A[2])


job = {}
jobid = 0
# analyze job tracker info
# get job id, job start time and finish time
for l in open(sys.argv[1]):
	A = l.split()
	if A[0] == "Job":
		for a in A:
			# store job information
			if a.startswith("JOBID"):
				job["id"] = int(getjobid(getvalue(a)))
				jobid = job["id"]
			if a.startswith("LAUNCH_TIME"):
				job["launch_time"] = (int(getvalue(a)))/1000
			if a.startswith("FINISH_TIME"):
				job["finish_time"] = (int(getvalue(a)))/1000


average = {}
hostinfo = {}
dnode = {}
min = 1272000000
for i in range(1,(int(sys.argv[2]))+1): 
	filename = "trace0506"+str(i)+".log"
	print filename
	machineid = i
	count = 0
	osinfo = {}
	# skip down machine
	if not hosts.has_key(i):
		continue

	# get active time range of this machine
	start = hosts[i]["start"]
	finish = hosts[i]["finish"]

	# init average list
	average[machineid] = {}
	average[machineid]["rbytes"] = 0 
	average[machineid]["wbytes"] = 0
	average[machineid]["msize"] = 0
	average[machineid]["rss"] = 0
	average[machineid]["cpu"] = 0.0

	hostinfo[machineid] = {}
	hostinfo[machineid]["rbytes"] = 0 
	hostinfo[machineid]["wbytes"] = 0
	hostinfo[machineid]["msize"] = 0
	hostinfo[machineid]["rss"] = 0
	hostinfo[machineid]["cpu"] = 0.0

	dnode[machineid] = {}
	dnode[machineid]["rbytes"] = 0 
	dnode[machineid]["wbytes"] = 0
	dnode[machineid]["msize"] = 0
	dnode[machineid]["rss"] = 0
	dnode[machineid]["cpu"] = 0.0


	for l in open(filename):
		A = l.split()
		id = 0
		time = 0
		rbytes = 0
		wbytes = 0
		msize = 0
		rss = 0
		cpu = 0.0
		type = {}
		for a in A:
			if a.startswith("property"):
				id = int(getjobid(getvalue(a)))
				if id == 99999999:
					type = getvalue(a)
					if type != "TaskTracker":
						id = jobid
			if a.startswith("walltime"):
				time = int(getvalue(a))
			if a.startswith("readbytes_thput="):
				rbytes = float(getvalue(a))
			if a.startswith("writebytes_thput="):
				wbytes = float(getvalue(a))
			if a.startswith("VmSize"):
				msize = int(getvalue(a))
			if a.startswith("VmRSS"):
				rss = int(getvalue(a))
			if a.startswith("cpu_per"):
				cpu = float(getvalue(a))
				if cpu > 8.0:
					cpu = 0.0

		# check job id (only host info and job info can pass)
		if id != jobid :
			continue

		# check active time range
		if (time < start-4) or (time > finish + 4):
			continue

		if type != "host":
			# record per machine per time slot information 
			if not osinfo.has_key(time):
				osinfo[time] = {}
				count += 1
				osinfo[time]["rbytes"] = 0 
				osinfo[time]["wbytes"] = 0
				osinfo[time]["msize"] = 0
				osinfo[time]["rss"] = 0
				osinfo[time]["cpu"] = 0.0

			# record in per machine temporal list
			osinfo[time]["rbytes"] += (rbytes/1024)
			osinfo[time]["wbytes"] += (wbytes/1024)
			osinfo[time]["msize"] += (msize/1024)
			osinfo[time]["rss"] += (rss/1024)
			osinfo[time]["cpu"] += (cpu*100/8)

			# record in machine average list
			average[machineid]["rbytes"] += (rbytes/1024)
			average[machineid]["wbytes"] += (wbytes/1024)
			average[machineid]["msize"] += (msize/1024)
			average[machineid]["rss"] += (rss/1024)
			average[machineid]["cpu"] += (cpu*100/8)
		elif type == "host":
			# record average host info
			hostinfo[machineid]["rbytes"] += (rbytes/1024)
			hostinfo[machineid]["wbytes"] += (wbytes/1024)
			hostinfo[machineid]["msize"] += (msize/1024)
			hostinfo[machineid]["rss"] += (rss/1024)
			hostinfo[machineid]["cpu"] += (cpu*100/8)
		else:
			dnode[machineid]["rbytes"] += (rbytes/1024)
			dnode[machineid]["wbytes"] += (wbytes/1024)
			dnode[machineid]["msize"] += (msize/1024)
			dnode[machineid]["rss"] += (rss/1024)
			dnode[machineid]["cpu"] += (cpu*100/8)



	# end of reading the log file

	# calculate per machine average value
	if count != 0:
		print count
		average[machineid]["rbytes"] /= count 
		average[machineid]["wbytes"] /= count 
		average[machineid]["msize"]  /= count
		average[machineid]["rss"]  /= count
		average[machineid]["cpu"]  /= count

		hostinfo[machineid]["rbytes"] /= count 
		hostinfo[machineid]["wbytes"] /= count 
		hostinfo[machineid]["msize"]  /= count
		hostinfo[machineid]["rss"]  /= count
		hostinfo[machineid]["cpu"]  /= count

		dnode[machineid]["rbytes"] /= count 
		dnode[machineid]["wbytes"] /= count 
		dnode[machineid]["msize"]  /= count
		dnode[machineid]["rss"]  /= count
		dnode[machineid]["cpu"]  /= count


			
	# save per machine per time slot information
	mf = open(str(i)+".dat", 'w')
	for k,v in osinfo.items():
		mf.write(str(k-min)+' '+ str(v["rbytes"])+' '+ str(v["wbytes"])+' '+ str(v["msize"])+' '+ str(v["rss"]) +' '+ str(v["cpu"])+'\n' )
	mf.close()

# save per machine average information
mf = open('average.dat', 'w')
for k,v in average.items():
	mf.write(str(k)+' '+ str(v["rbytes"]+dnode[k]["rbytes"])+' '+ str(v["wbytes"]+dnode[k]["wbytes"])+' '+ str(v["msize"]+dnode[k]["msize"])+' '+ str(v["rss"]+dnode[k]["rss"]) +' '+ str(v["cpu"]+dnode[k]["cpu"])+' '+ str(hostinfo[k]["rbytes"])+' '+ str(hostinfo[k]["wbytes"])+' '+ str(hostinfo[k]["msize"])+' '+ str(hostinfo[k]["rss"]) +' '+ str(hostinfo[k]["cpu"])+'\n' )
mf.close()



